Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Restart storage alert confirmation across missing observations
pulse-triage[bot]23355440ace3
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex 6d4a6def25..081be8700b 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -3876,3 +3876,26 @@ It covers both families sharing a disk, interruption/reconfirmation, partialrule disablement, re-enablement, legacy/unknown causes, filesystem and hostisolation, removal and node-link ownership. These are source lifecycle controls,not collector, smartctl, installed notification or native disk-recovery proof.++### Storage observation gaps restart confirmation and timing++An absent or `unknown` storage connectivity status interrupts activation,+recovery confirmations and pending offline intent grace without resolving an+existing outage or changing its acknowledgement. Capacity is independently+observable while connectivity is unknown. Missing/unconfirmed-zero capacity,+negative usage and offline/unavailable storage interrupt capacity activation,+intent grace and recovery timing; they do not supply healthy capacity evidence.+A confirmed empty store still supplies a genuine zero-usage recovery reading.+Fresh observations must satisfy the existing full confirmation/delay after a+gap. Interruption is scoped to the storage's canonical identity and applied to+both live and shadow reducers; interrupted intent is checkpointed for restart.++`TestStorageConnectivityObservationGaps` and+`TestStorageConnectivityGapRestartsIntentGrace` exercise the real storage+entry point, acknowledged occurrence retention, neighbour isolation, fresh+reconfirmation and empty intent checkpoint. Storage routes in+`TestMetricObservationGapRestartsActivation` and+`TestMetricObservationGapRestartsRecovery` cover capacity gaps and explicit+intent grace. Existing restart, empty-capacity and predictive-capacity controls+remain required. These source controls are not native storage recovery or+installed destination-delivery evidence.diff --git a/internal/alerts/metric_observation_continuity_test.go b/internal/alerts/metric_observation_continuity_test.goindex 6ba596e8da..ffef188997 100644--- a/internal/alerts/metric_observation_continuity_test.go+++ b/internal/alerts/metric_observation_continuity_test.go@@ -108,6 +108,25 @@ func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, st}m.evaluateUnifiedMetrics(&UnifiedResourceInput{ID: id, Type: "vm", Name: "guest", Memory: sample}, ThresholdConfig{Memory: threshold}, nil)}+ case "storage":+ id, metric = "site:node:backups", "usage"+ return id, metric, func(value float64, missing bool) {+ storage := models.Storage{ID: id, Name: "backups", Node: "node", Instance: "site", Status: "online",+ Total: 1000, Used: int64(value * 10), Free: int64(1000 - value*10), Usage: value}+ if missing {+ switch gap {+ case "offline", "unavailable":+ storage.Status = gap+ case "negative":+ storage.Usage = -1+ case "unconfirmed-zero":+ storage.Usage = 0+ default:+ storage.Total, storage.Used, storage.Free, storage.Usage = 0, 0, 0, 0+ }+ }+ m.CheckStorage(storage)+ }case "host":id = "agent:host-105"return id, metric, func(value float64, missing bool) {@@ -184,11 +203,14 @@ func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, stfunc TestMetricObservationGapRestartsActivation(t *testing.T) {for _, explicit := range []bool{false, true} {- for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired", "host-disk-temperature"} {+ for _, route := range []string{"legacy", "canonical", "unified", "host", "storage", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired", "host-disk-temperature"} {gaps := []string{"missing"}if route == "legacy" || route == "canonical" {gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}}+ if route == "storage" {+ gaps = []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"}+ }if route == "host-disk-temperature" {gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}}@@ -237,11 +259,14 @@ func TestMetricObservationGapRestartsActivation(t *testing.T) {}func TestMetricObservationGapRestartsRecovery(t *testing.T) {- for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "host-disk-temperature"} {+ for _, route := range []string{"legacy", "canonical", "unified", "host", "storage", "guest-memory", "host-disk-temperature"} {gaps := []string{"missing"}if route == "legacy" || route == "canonical" {gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}}+ if route == "storage" {+ gaps = []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"}+ }if route == "host-disk-temperature" {gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}}diff --git a/internal/alerts/storage.go b/internal/alerts/storage.goindex ccbf35a0f2..3bb2ac80ef 100644--- a/internal/alerts/storage.go+++ b/internal/alerts/storage.go@@ -88,6 +88,8 @@ func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend Ca} else if connectivityStatus != "" && connectivityStatus != "unknown" {// Clear any existing offline alert if storage is back onlinem.clearStorageOfflineAlert(storage)+ } else {+ m.interruptStorageConnectivityRun(storage.ID)}// Missing connectivity evidence must not count as a healthy observation.// Capacity and pool health below remain independently observable.@@ -110,6 +112,8 @@ func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend CaconfirmedEmpty := storage.Usage == 0 && storage.Total > 0 && storage.Used == 0 && storage.Free == storage.Totalif connectivityStatus != "offline" && connectivityStatus != "unavailable" && (storage.Usage > 0 || confirmedEmpty) {m.evaluateStorageCapacity(storage, thresholds, trend)+ } else {+ m.interruptMetricRunIDs(storage.ID, canonicalMetricSpecID(storage.ID, "usage"), canonicalMetricStateID(storage.ID, "usage"))}// Check ZFS pool status if this is ZFS storage@@ -120,6 +124,21 @@ func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend Ca}}+// A missing status is not a healthy poll, nor another outage confirmation.+// Preserve a firing occurrence while restarting both consecutive-observation+// runs and any pending intent grace. Capacity remains independently observable.+func (m *Manager) interruptStorageConnectivityRun(resourceID string) {+ m.mu.Lock()+ for _, state := range m.mirrorStatesNoLock() {+ state.InterruptDiscreteRun(resourceID, canonicalConnectivitySpecID(resourceID))+ }+ intentChanged := m.clearIntentPendingNoLock(canonicalConnectivityStateID(resourceID))+ m.mu.Unlock()+ if intentChanged {+ m.saveActiveAlertsAsync("storage connectivity observation gap")+ }+}+// clearStorageZFSAlerts removes ZFS pool/device alerts for a storage that no// longer carries an attached ZFS pool. checkZFSPoolHealth can only clear its// own alerts while the pool stays attached, so a storage whose attachment goesdiff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.goindex 53aaa6e226..1b05799eb7 100644--- a/internal/alerts/threshold_resolution_shared_test.go+++ b/internal/alerts/threshold_resolution_shared_test.go@@ -1,7 +1,11 @@package alertsimport (+ "encoding/json""math"+ "os"+ "path/filepath"+ "reflect""testing""time"@@ -1701,3 +1705,118 @@ func TestReleaseCanonicalMetricAlertClearsIntentPending(t *testing.T) {t.Fatalf("expected releasing the metric to clear its intent pending state")}}++// Missing storage observations break confirmation runs, not the occurrence.+// Exercise the public evaluator and both live/shadow reducers, not seeded state.+func TestStorageConnectivityObservationGaps(t *testing.T) {+ for _, gap := range []string{"", "unknown", " UNKNOWN "} {+ t.Run("activation/"+gap, func(t *testing.T) {+ m := newShadowFeedManager(t)+ s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}+ other := s+ other.ID = "storage-ab"+ id := canonicalConnectivityStateID(s.ID)+ m.CheckStorage(s)+ m.CheckStorage(other)+ s.Status = gap+ m.CheckStorage(s)+ s.Status = "unavailable"+ m.CheckStorage(s)+ if testHasActiveAlert(t, m, id) {+ t.Fatal("non-consecutive outage observations fired storage alert")+ }+ m.CheckStorage(other)+ testRequireActiveAlert(t, m, canonicalConnectivityStateID(other.ID))+ m.CheckStorage(s)+ testRequireActiveAlert(t, m, id)+ if m.ShadowDivergences() != 0 {+ t.Fatal("storage interruption diverged from shadow")+ }+ })+ t.Run("recovery/"+gap, func(t *testing.T) {+ m := newShadowFeedManager(t)+ s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}+ id := canonicalConnectivityStateID(s.ID)+ for range 2 {+ m.CheckStorage(s)+ }+ if err := m.AcknowledgeAlert(id, "operator"); err != nil {+ t.Fatal(err)+ }+ s.Status = "available"+ m.CheckStorage(s)+ before := testRequireActiveAlert(t, m, id).Clone()+ s.Status = gap+ m.CheckStorage(s)+ if !reflect.DeepEqual(before, testRequireActiveAlert(t, m, id)) || m.GetResolvedAlert(id) != nil {+ t.Fatal("missing connectivity changed acknowledged occurrence")+ }+ s.Status = "available"+ m.CheckStorage(s)+ if !testHasActiveAlert(t, m, id) {+ t.Fatal("non-consecutive healthy observations resolved storage outage")+ }+ m.CheckStorage(s)+ if testHasActiveAlert(t, m, id) {+ t.Fatal("fresh consecutive healthy observations failed to recover")+ }+ resolved := m.GetResolvedAlert(id)+ if resolved == nil || !resolved.StartTime.Equal(before.StartTime) {+ t.Fatal("confirmed recovery lost original occurrence")+ }+ if m.ShadowDivergences() != 0 {+ t.Fatal("storage recovery diverged from shadow")+ }+ })+ }+}++func TestStorageConnectivityGapRestartsIntentGrace(t *testing.T) {+ m := newShadowFeedManager(t)+ var tick time.Duration+ m.intentClock = func() time.Duration { return tick }+ doc := NewAlertIntentPolicyDocument()+ doc.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{GraceSeconds: intPointer(60)}+ if err := m.LoadIntentPolicies(doc); err != nil {+ t.Fatal(err)+ }+ s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}+ id := canonicalConnectivityStateID(s.ID)+ m.CheckStorage(s)+ tick = 40 * time.Second+ m.CheckStorage(s)+ s.Status = "unknown"+ m.CheckStorage(s)+ m.mu.RLock()+ _, pending := m.intentPending[id]+ _, ticking := m.intentRuntimeTicks[id]+ m.mu.RUnlock()+ if pending || ticking {+ t.Error("missing status retained intent grace bookkeeping")+ }+ if err := m.SaveActiveAlerts(); err != nil {+ t.Fatal(err)+ }+ data, err := os.ReadFile(filepath.Join(m.getAlertsDir(), intentPendingFileName))+ if err != nil {+ t.Fatal(err)+ }+ var saved []IntentPendingState+ if err := json.Unmarshal(data, &saved); err != nil || len(saved) != 0 {+ t.Fatalf("interrupted intent remained checkpointed: %s, %v", data, err)+ }+ s.Status = "offline"+ for _, next := range []time.Duration{70, 90, 129} {+ tick = next * time.Second+ m.CheckStorage(s)+ if testHasActiveAlert(t, m, id) {+ t.Fatal("intent grace accrued across missing storage status")+ }+ }+ tick = 130 * time.Second+ m.CheckStorage(s)+ testRequireActiveAlert(t, m, id)+ if m.ShadowDivergences() != 0 {+ t.Fatal("intent interruption diverged from shadow")+ }+}